package lib::Wget;

# This file is part of noplagia.
#
# noplagia is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# noplagia is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# noplagia; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
# Suite 330, Boston, MA  02111-1307 USA

use strict;
use WWW::Mechanize;
use Data::Dumper;
use lib::Context;
use File::Spec;
use lib::TextTreatment;
use File::Temp qw/ tempfile tempdir /;
use Encode;
require Exporter;
our ($VERSION,@ISA,@EXPORT,$context);

$VERSION = 0.01;

=head1 NAME

lib::Wget - fouille des sites web

=head1 SYNOPSIS

    use lib::Wget;
    TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO

=head1 DESCRIPTION

    Ce module permet de fouiller des sites web pour trouver des texte plagies

=head1 FUNCTIONS

=over 2

=cut

@ISA    = qw(Exporter);
@EXPORT = qw(
    &wget
);

=item wget

    my $result = wget(lib::ResultTreatment::result_traetment(\@result));
    Get all web site content from links stored into the input hash.

=cut

use Encode;
sub wget($) {
    my $simili = get_pref('dl_simili');
    my $in  = shift;
    my $moyenne = shift(@$in);
    my $mech = WWW::Mechanize->new();
    # Create a request
    foreach (@$in){
        my $urls = $_->{'urls'};
        foreach(@$urls){
            next if ($_->{match} < $simili);
            $mech->agent_alias('Linux Mozilla');
            $mech->get( $_->{'url'} );
            if ($mech->ct() eq 'application/pdf'){
                my ($fh, $filename) = tempfile();
                print $fh $mech->content( format => 'text' );
                my $text =  `pdftotext -enc UTF-8 -eol unix $filename /dev/stdout`;
                $fh->close();
                $_->{'content'} = encode('utf8', set_noPunctuation($text));
            }else{
                $_->{'content'} = encode('utf8', set_noPunctuation($mech->content( format => 'text' )));
            }
        }
    }

    return $in;
}


END { }
1;
__END__

=back

=head1 AUTEUR

    noplagia dev team

    http://noplagia.org

=cut
